# Scrapy项目设置
BOT_NAME = 'forex_spider'

SPIDER_MODULES = ['forex_spider.spiders']
NEWSPIDER_MODULE = 'forex_spider.spiders'

# 遵守robots.txt规则
ROBOTSTXT_OBEY = False

# 并发设置
CONCURRENT_REQUESTS = 1
DOWNLOAD_DELAY = 2  # 2秒延迟，避免请求过快

# 管道设置
ITEM_PIPELINES = {
    'forex_spider.pipelines.ValidationPipeline': 100,  # 数据验证
    'forex_spider.pipelines.ConsolePipeline': 200,    # 控制台输出
    'forex_spider.pipelines.SQLitePipeline': 300,     # 数据库存储（推荐）
    # 'forex_spider.pipelines.MySQLPipeline': 300,    # MySQL存储（备用）
}

# 下载中间件
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
}

# 重试设置
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 400, 403, 404, 408]

# User-Agent设置
USER_AGENT = (
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
    '(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
)

# 日志设置
LOG_LEVEL = 'INFO'
LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'

# MySQL配置（备用）
MYSQL_URI = 'mysql+pymysql://root:@localhost:3306/forex_db'

# 自动限速扩展
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1
AUTOTHROTTLE_MAX_DELAY = 10